﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Net;
using System.IO;

namespace WebImageSpider
{
    public class CatSearchEngine
    {
        private SearchWebType searchWebType;
        private int baiduCompleteCount = 0;
        private int baiduFailCount = 0;
        private int baiduSubmissionCount = 0;

        private readonly int baiduCount;
        private readonly int googleCount;
        private int googleCompleteCount = 0;
        private int googleFailCount = 0;
        private int googleSubmissionCount = 0;
        private int pageIndex = 0;
        private string keyWord;
        private const string baiduBaseURL = "http://image.baidu.com/i?z=0&cl=2&ct=201326592&sn=&lm=-1&cm=1&sc=0&bu=&rn=20&tn=baiduimage&word=";//%C3%B5%B9%E5&pn=16;
        //                                    http://images.google.cn/images?hl=zh-CN&source=hp&q=%E6%9E%97%E5%BF%97%E7%8E%B2&gbv=2&aq=f&oq=
        private const string googleBaseURL = "http://images.google.cn/images?q=!Word&svnum=10&hl=zh-CN&lr=&newwindow=1&start=";//&sa=N";

        public CatSearchEngine(SearchWebType type, string _keyWord, int num)
        {
            keyWord = _keyWord;
            searchWebType = type;
            if (type == SearchWebType.All)
            {
                googleCount = baiduCount = num;
            }
            else if (type == SearchWebType.Baidu)
            {
                baiduCount = num;
            }
            else if (type == SearchWebType.Google)
            {
                googleCount = num;
            }

        }

        public int BaiduCompleteCount
        {
            get
            {
                lock (this)
                {
                    return baiduCompleteCount;
                }
            }
        }

        public int GoogleCompleteCount
        {
            get
            {
                lock (this)
                {
                    return googleCompleteCount;
                }
            }
        }

        public int IncreaseBaiduComleteCountInterlocked()
        {
            lock (this)
            {
                baiduCompleteCount++;
                //Debug.Assert(baiduCompleteCount + baiduFailCount <= baiduSubmissionCount);
                //if (baiduCompleteCount >= baiduCount)
                //    SqlUpdateWordStatus(keyWord, KeyWordStatus.Completed);
                return baiduCompleteCount;
            }
        }

        public int IncreaseGoogleCompleteCountInterlocked()
        {
            lock (this)
            {
                googleCompleteCount++;
                //Debug.Assert(googleCompleteCount + googleFailCount <= googleSubmissionCount);
                //if (googleCompleteCount >= googleCount)
                //    SqlUpdateWordStatus(keyWord, KeyWordStatus.Completed);
                return googleCompleteCount;
            }
        }

        public int IncreaseBaiduFailedCountInterlocked()
        {
            lock (this)
            {
                //Debug.Assert(baiduCompleteCount + baiduFailCount <= baiduSubmissionCount);
                baiduFailCount++;
                return baiduFailCount;
            }
        }

        public int IncreaseGoogleFailedCountInterlocked()
        {
            lock (this)
            {
                //Debug.Assert(googleCompleteCount + googleFailCount <= googleSubmissionCount);
                googleFailCount++;
                return googleFailCount;
            }
        }

        public int IncreaseBaiduSubmissionInterlocked()
        {
            lock (this)
            {
                //Debug.Assert(baiduCompleteCount + baiduFailCount <= baiduSubmissionCount);
                baiduSubmissionCount++;
                return baiduSubmissionCount;
            }
        }

        public int IncreaseGoogleSubmissionInterlocked()
        {
            lock (this)
            {
                //Debug.Assert(googleCompleteCount + googleFailCount <= googleSubmissionCount);
                googleSubmissionCount++;
                return googleSubmissionCount;
            }
        }

        public int GetAndIncreasePageIndex()
        {
            lock (this)
            {
                pageIndex++;
                return pageIndex;
            }
        }

        public bool DoSearch()
        {
            //if (!SqlUpdateWordStatus(keyWord, KeyWordStatus.Running))
            //    return false;
            if ((int)(searchWebType & SearchWebType.Baidu) > 0)
            {
                GetBaiDuPage(keyWord);
            }
            if ((int)(searchWebType & SearchWebType.Google) > 0)
            {
                GetGooglePage(keyWord);
            }
            //int availableWorkerThread;
            //int availableCompletePort;
            //int maxWorkerThread;
            //int maxCompletePort;
            //Thread.Sleep(2000);
            //ThreadPool.GetAvailableThreads(out availableWorkerThread, out availableCompletePort);
            //ThreadPool.GetMaxThreads(out maxWorkerThread, out maxCompletePort);

            //while (availableWorkerThread < maxWorkerThread)
            //{
            //    Thread.Sleep(5000);
            //    ThreadPool.GetAvailableThreads(out availableWorkerThread, out availableCompletePort);
            //    ThreadPool.GetMaxThreads(out maxWorkerThread, out maxCompletePort);
            //}
            //SqlUpdateWordStatus(keyWord, KeyWordStatus.Completed);
            return true;
        }

        private void GetBaiDuPage(string keyWord)
        {
            const int pageItemCount = 20;
            //v1:const string pattern = "http://image.baidu.com/ir?t=1&u=";
            //v2:const string pattern = "img src=http://t";
            //v3:
            const string pattern = "<img name='pn";

            Encoding Encoder = System.Text.Encoding.GetEncoding("GB2312");
            byte[] ba = Encoder.GetBytes(keyWord);
            StringBuilder url = new StringBuilder(baiduBaseURL);
            foreach (byte b in ba)
            {
                url.Append("%");
                url.Append(b.ToString("X2"));
            }
            url.Append("&pn=");
            int pageCount = baiduCount / pageItemCount;
            if (pageCount <= 0)
                pageCount = 1;
            int i = 0;
            while (BaiduCompleteCount < baiduCount)
            {
                //for (int i = 0; i < pageCount; i++)
                //{
                //cout << "completed pic nums = "<<BaiduCompleteCount << endl;
                Console.WriteLine(" {0} completed pic nums = {1}.", keyWord, BaiduCompleteCount);

                while (baiduSubmissionCount - baiduFailCount > baiduCount + 30)
                {
                    if (BaiduCompleteCount >= baiduCount)
                        return;
                    Thread.Sleep(1000);
                }
                HttpWebResponse response = null;
                try
                {
                    response = HttpService.GetHttpWebResponse(url.ToString() + (i * pageItemCount).ToString());
                   
                }
                catch (Exception)
                {
                    if (response != null)
                        response.Close();
                    continue;
                }
                bool b = (response == null);
                if (response != null && response.ContentType.StartsWith("text/html"))
                {
                    StreamReader inStream = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("GB2312"));
                    string text = inStream.ReadToEnd();
                    response.Close();
                    inStream.Close();
                    int startIndex = 0;
                    int stopIndex = 0;
                    int subCount = 0;

                    //int realstart = startIndex = text.IndexOf("if(imgNum<21)",1);

                    // ?  "img name='pn"  20090926
                    //int realstart = startIndex = text.IndexOf("img src=http://t", 1);
                    int realstart = startIndex = text.IndexOf("<img name='pn", 1);
                    while ((startIndex = text.IndexOf(pattern, startIndex + 1)) != -1)
                    {
                        int baiduPicStart = text.IndexOf("http://t", realstart + 1);
                        int baiduPicStop = text.IndexOf("\"><", baiduPicStart + 1);
                        string baiduPic = text.Substring(baiduPicStart,baiduPicStop - baiduPicStart);
                        //baiduPic = TransferCode(baiduPic, Encoding.GetEncoding("GB2312"));

                        //stopIndex = text.IndexOf("imgNum++", startIndex);
                        stopIndex = text.IndexOf("</dl>", startIndex);

                        realstart = stopIndex;

                        string link = text.Substring(startIndex, stopIndex - startIndex);
                        string[] subLink = link.Split(new char[] { '&' });
                        if (subLink != null)
                        {
                            //string picUrl = subLink[1].Substring(2);
                            string picUrl = subLink[3].Substring(2);
                            //picUrl = TransferCode(picUrl, Encoding.GetEncoding("GB2312"));
                            //string htmlUrl = subLink[2].Substring(2);
                            string htmlUrl = subLink[4].Substring(2);
                            //htmlUrl = TransferCode(htmlUrl, Encoding.GetEncoding("GB2312"));
                            UrlItem ui = new UrlItem(baiduPic,picUrl, htmlUrl, keyWord, GetAndIncreasePageIndex(), this, SearchWebType.Baidu, subCount);
                            ++subCount;
                            ThreadPool.QueueUserWorkItem(HttpService.GetURLFun, ui);
                            IncreaseBaiduSubmissionInterlocked();
                        }

                    }
                    if (stopIndex == 0)
                    {
                        //SqlUpdateWordStatus(keyWord, KeyWordStatus.Completed);
                        return;
                    }
                }
                ++i;
            }

        }
        private long GetGoogleRsltNum_Text(string key)
        {
            //     http://www.google.cn/search?hl=zh-CN&q=tiger&meta=
            string strFor = "http://www.google.cn/search?hl=zh-CN&q=";
            string strAfter = "&meta=";
            string strAll = strFor + key + strAfter;
            HttpWebResponse response = HttpService.GetHttpWebResponse(strAll);
            StreamReader inStream = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
            string text = inStream.ReadToEnd();


            int pos = 0;
            for (int i = 0; i < 5; ++i)
            {
                pos = text.IndexOf("<table", pos + 1);
            }

            pos = text.IndexOf("<b>", pos);
            pos = text.IndexOf("<b>", pos + 1);

            int end = text.IndexOf("<", pos + 1);

            string strNum = text.Substring(pos + 3, end - pos - 3);
            char[] delimiter = { ',' };
            String[] strs = strNum.Split(delimiter);

            strNum = "";
            foreach (string str in strs)
                strNum += str;

            return long.Parse(strNum);
        }
        private long GetGoogleRsltNum_Pic(string key)
        {
            string strFor = "http://images.google.cn/images?hl=zh-CN&q=";
            string strAfter = "&gbv=2";
            string strAll = strFor + key + strAfter;
            HttpWebResponse response = HttpService.GetHttpWebResponse(strAll);
            StreamReader inStream = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
            string text = inStream.ReadToEnd();


            int pos = 0;
            for (int i = 0; i < 5; ++i)
            {
                pos = text.IndexOf("<table", pos + 1);
            }

            pos = text.IndexOf("<b>", pos);
            pos = text.IndexOf("<b>", pos + 1);

            int end = text.IndexOf("<", pos + 1);

            string strNum = text.Substring(pos + 3, end - pos - 3);
            char[] delimiter = { ',' };
            String[] strs = strNum.Split(delimiter);

            strNum = "";
            foreach (string str in strs)
                strNum += str;

            return long.Parse(strNum);
        }



        private void GetGooglePage(string keyWord)
        {


            const int pageItemCount = 20;
            const string patternImage = "imgurl=";
            int patternImageLength = patternImage.Length;
            const string patternHtml = "imgrefurl=";
            int patternHtmlLength = patternHtml.Length;
            Encoding encoding = Encoding.UTF8;

            Encoding Encoder = System.Text.Encoding.UTF8;
            byte[] ba = Encoder.GetBytes(keyWord);
            StringBuilder keyCode = new StringBuilder();
            string googleUrl;
            foreach (byte b in ba)
            {
                keyCode.Append("%");
                keyCode.Append(b.ToString("X2"));
            }
            googleUrl = googleBaseURL.Replace("!Word", keyCode.ToString());
            int pageCount = googleCount / pageItemCount;
            if (pageCount <= 0)
                pageCount = 1;
            int i = 0;
            //目录名
            int subCount = 0;

            while (GoogleCompleteCount < googleCount)
            {
                //for (int i = 0; i < pageCount; i++)
                //{
                while (googleSubmissionCount - googleFailCount > googleCount + 30)
                {
                    if (GoogleCompleteCount >= googleCount)
                        return;
                    Thread.Sleep(1000);
                }
                HttpWebResponse response = null;
                try
                {
                    int pc = i * pageItemCount;
                    //计算视觉属性
                    if(i==0)
                    {
                        /*long textNum = GetGoogleRsltNum_Text(keyWord);
                        // 图片的数目
                        long picNum = GetGoogleRsltNum_Pic(keyWord);

                        string visualPath = "F:\\AIA\\save\\visual.txt";
                        StreamWriter note = new StreamWriter(visualPath, true, System.Text.Encoding.Default);
                        double val = (double)picNum / textNum * 100;
                        note.WriteLine(val);
                        note.Close();*/

                    }
                    response = HttpService.GetHttpWebResponse(googleUrl + pc.ToString() + "&sa=N");
                }
                catch (Exception)
                {
                    if (response != null)
                        response.Close();
                    continue;
                }
                if (response != null && response.ContentType.StartsWith("text/html"))
                {

                    StreamReader inStream = new StreamReader(response.GetResponseStream(), encoding);
                    string text = inStream.ReadToEnd();

                    string head = text.Substring(60, 20);
                    if (head.Contains("UTF-8") || head.Contains("utf-8"))
                    {
                        encoding = Encoding.UTF8;
                        //  continue;
                    }
                    else
                        encoding = Encoding.GetEncoding("GB2312");
                    response.Close();
                    inStream.Close();
                    int startIndex = 0;
                    int stopIndex = 0;
                    //每个返回结果有20个图片
                    while ((startIndex = text.IndexOf(patternImage, startIndex + 1)) != -1)
                    {
                        string googlePic = "";
                        stopIndex = text.IndexOf("&", startIndex);
                        string picUrl = text.Substring(startIndex + patternImageLength, stopIndex - startIndex - patternImageLength);
                        picUrl = TransferCode(picUrl, encoding);

                        if ((startIndex = text.IndexOf(patternHtml, startIndex + 1)) == -1)
                            break;
                        stopIndex = text.IndexOf("&", startIndex);
                        string htmlUrl = text.Substring(startIndex + patternHtmlLength, stopIndex - startIndex - patternHtmlLength);
                        htmlUrl = TransferCode(htmlUrl, encoding);

                        startIndex = text.IndexOf("/images?", startIndex);
                        stopIndex = text.IndexOf("http://",startIndex);
                        googlePic = "http://images.google.cn" + TransferCode(text.Substring(startIndex, stopIndex - startIndex), encoding) + picUrl;

                        UrlItem ui = new UrlItem(googlePic,picUrl, htmlUrl, keyWord, GetAndIncreasePageIndex(), this, SearchWebType.Google,subCount);
                        ++subCount;
                        ThreadPool.QueueUserWorkItem(HttpService.GetURLFun, ui);
                        IncreaseGoogleSubmissionInterlocked();
                    }
                    if (stopIndex == 0)
                    {
                        ////SqlUpdateWordStatus(keyWord, KeyWordStatus.Completed);
                        return;
                    }
                }
                ++i;
            }
        }

        //private bool SqlUpdateWordStatus(string keyWord, KeyWordStatus kws)
        //{
        //    int rtn;
        //    lock (FormConfig.sqlConnection)
        //    {
        //        SqlCommand sqlCmd = FormConfig.sqlConnection.CreateCommand();
        //        if (kws == KeyWordStatus.Running)
        //        {
        //            sqlCmd.CommandText = "UPDATE WordTab SET wFlag = 1 WHERE word = '" + keyWord + "' AND wFlag = 0";
        //        }
        //        else
        //        {
        //            sqlCmd.CommandText = "UPDATE WordTab SET wFlag = 2 WHERE word = '" + keyWord + "'";
        //        }
        //        rtn = sqlCmd.ExecuteNonQuery();
        //        sqlCmd.Dispose();
        //    }
        //    return rtn != 0;
        //}

        public static string TransferCode(string code, Encoding encoding)
        {
            string[] segments = code.Split(new char[] { '%' });
            if (segments.Length == 1)
                return code;
            byte[] c = new byte[1];
            string subs;
            StringBuilder sb = new StringBuilder();
            sb.Append(segments[0]);
            for (int i = 1; i < segments.Length; i++)
            {
                subs = segments[i].Substring(0, 2);
                try
                {
                    c[0] = Convert.ToByte(subs, 16);
                }
                catch (Exception)
                {
                    return code;
                }
                sb.Append(encoding.GetString(c));
                if (segments[i].Length > 2)
                    sb.Append(segments[i].Substring(2));
            }
            return sb.ToString();
        }
    }
}

enum KeyWordStatus
{
    Running,
    Completed,
}
